

<!DOCTYPE html>
<html class="writer-html5" lang="zh" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>使用浏览器的开发人员工具进行抓取 &mdash; Scrapy 2.3.0 文档</title>
  

  
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster.custom.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster.bundle.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-shadow.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-punk.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-noir.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-light.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-borderless.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/micromodal.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/sphinx_rtd_theme.css" type="text/css" />

  
  
  
  

  
  <!--[if lt IE 9]>
    <script src="../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script src="../_static/jquery.js"></script>
        <script src="../_static/underscore.js"></script>
        <script src="../_static/doctools.js"></script>
        <script src="../_static/language_data.js"></script>
        <script src="../_static/js/hoverxref.js"></script>
        <script src="../_static/js/tooltipster.bundle.min.js"></script>
        <script src="../_static/js/micromodal.min.js"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    
    <link rel="index" title="索引" href="../genindex.html" />
    <link rel="search" title="搜索" href="../search.html" />
    <link rel="next" title="选择动态加载的内容" href="dynamic-content.html" />
    <link rel="prev" title="宽爬行" href="broad-crawls.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../index.html" class="icon icon-home" alt="Documentation Home"> Scrapy
          

          
          </a>

          
            
            
              <div class="version">
                2.3
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">第一步</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../intro/overview.html">Scrapy一目了然</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/install.html">安装指南</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/tutorial.html">Scrapy 教程</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/examples.html">实例</a></li>
</ul>
<p class="caption"><span class="caption-text">基本概念</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="commands.html">命令行工具</a></li>
<li class="toctree-l1"><a class="reference internal" href="spiders.html">蜘蛛</a></li>
<li class="toctree-l1"><a class="reference internal" href="selectors.html">选择器</a></li>
<li class="toctree-l1"><a class="reference internal" href="items.html">项目</a></li>
<li class="toctree-l1"><a class="reference internal" href="loaders.html">项目加载器</a></li>
<li class="toctree-l1"><a class="reference internal" href="shell.html">Scrapy shell</a></li>
<li class="toctree-l1"><a class="reference internal" href="item-pipeline.html">项目管道</a></li>
<li class="toctree-l1"><a class="reference internal" href="feed-exports.html">Feed 导出</a></li>
<li class="toctree-l1"><a class="reference internal" href="request-response.html">请求和响应</a></li>
<li class="toctree-l1"><a class="reference internal" href="link-extractors.html">链接提取器</a></li>
<li class="toctree-l1"><a class="reference internal" href="settings.html">设置</a></li>
<li class="toctree-l1"><a class="reference internal" href="exceptions.html">例外情况</a></li>
</ul>
<p class="caption"><span class="caption-text">内置服务</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="logging.html">登录</a></li>
<li class="toctree-l1"><a class="reference internal" href="stats.html">统计数据集合</a></li>
<li class="toctree-l1"><a class="reference internal" href="email.html">发送电子邮件</a></li>
<li class="toctree-l1"><a class="reference internal" href="telnetconsole.html">远程登录控制台</a></li>
<li class="toctree-l1"><a class="reference internal" href="webservice.html">Web服务</a></li>
</ul>
<p class="caption"><span class="caption-text">解决具体问题</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../faq.html">常见问题</a></li>
<li class="toctree-l1"><a class="reference internal" href="debug.html">调试spiders</a></li>
<li class="toctree-l1"><a class="reference internal" href="contracts.html">蜘蛛合约</a></li>
<li class="toctree-l1"><a class="reference internal" href="practices.html">常用做法</a></li>
<li class="toctree-l1"><a class="reference internal" href="broad-crawls.html">宽爬行</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">使用浏览器的开发人员工具进行抓取</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#caveats-with-inspecting-the-live-browser-dom">检查实时浏览器DOM时的注意事项</a></li>
<li class="toctree-l2"><a class="reference internal" href="#inspecting-a-website">查看网站</a></li>
<li class="toctree-l2"><a class="reference internal" href="#the-network-tool">网络工具</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="dynamic-content.html">选择动态加载的内容</a></li>
<li class="toctree-l1"><a class="reference internal" href="leaks.html">调试内存泄漏</a></li>
<li class="toctree-l1"><a class="reference internal" href="media-pipeline.html">下载和处理文件和图像</a></li>
<li class="toctree-l1"><a class="reference internal" href="deploy.html">部署蜘蛛</a></li>
<li class="toctree-l1"><a class="reference internal" href="autothrottle.html">AutoThrottle 扩展</a></li>
<li class="toctree-l1"><a class="reference internal" href="benchmarking.html">标杆管理</a></li>
<li class="toctree-l1"><a class="reference internal" href="jobs.html">作业：暂停和恢复爬行</a></li>
<li class="toctree-l1"><a class="reference internal" href="coroutines.html">协同程序</a></li>
<li class="toctree-l1"><a class="reference internal" href="asyncio.html">asyncio</a></li>
</ul>
<p class="caption"><span class="caption-text">扩展Scrapy</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="architecture.html">体系结构概述</a></li>
<li class="toctree-l1"><a class="reference internal" href="downloader-middleware.html">下载器中间件</a></li>
<li class="toctree-l1"><a class="reference internal" href="spider-middleware.html">蜘蛛中间件</a></li>
<li class="toctree-l1"><a class="reference internal" href="extensions.html">扩展</a></li>
<li class="toctree-l1"><a class="reference internal" href="api.html">核心API</a></li>
<li class="toctree-l1"><a class="reference internal" href="signals.html">信号</a></li>
<li class="toctree-l1"><a class="reference internal" href="exporters.html">条目导出器</a></li>
</ul>
<p class="caption"><span class="caption-text">其余所有</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../news.html">发行说明</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributing.html">为 Scrapy 贡献</a></li>
<li class="toctree-l1"><a class="reference internal" href="../versioning.html">版本控制和API稳定性</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">Scrapy</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
        
      <li>使用浏览器的开发人员工具进行抓取</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="using-your-browser-s-developer-tools-for-scraping">
<span id="topics-developer-tools"></span><h1>使用浏览器的开发人员工具进行抓取<a class="headerlink" href="#using-your-browser-s-developer-tools-for-scraping" title="永久链接至标题">¶</a></h1>
<p>下面是关于如何使用浏览器的开发人员工具来简化抓取过程的一般指南。现在几乎所有浏览器都内置了 <a class="reference external" href="https://en.wikipedia.org/wiki/Web_development_tools">Developer Tools</a> 尽管我们将在本指南中使用firefox，但这些概念适用于任何其他浏览器。</p>
<p>在本指南中，我们将介绍通过抓取从浏览器的开发人员工具中使用的基本工具 <a class="reference external" href="http://quotes.toscrape.com">quotes.toscrape.com</a> .</p><script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>
<ins class="adsbygoogle"
     style="display:block; text-align:center;"
     data-ad-layout="in-article"
     data-ad-format="fluid"
     data-ad-client="ca-pub-1466963416408457"
     data-ad-slot="8850786025"></ins>
<script>
     (adsbygoogle = window.adsbygoogle || []).push({});
</script>
<div class="section" id="caveats-with-inspecting-the-live-browser-dom">
<span id="topics-livedom"></span><h2>检查实时浏览器DOM时的注意事项<a class="headerlink" href="#caveats-with-inspecting-the-live-browser-dom" title="永久链接至标题">¶</a></h2>
<p>由于开发人员工具在一个活动的浏览器DOM上运行，所以在检查页面源代码时，您实际上看到的不是原始的HTML，而是应用了一些浏览器清理和执行javascript代码后修改的HTML。尤其是火狐，以添加 <code class="docutils literal notranslate"><span class="pre">&lt;tbody&gt;</span></code> 元素到表。另一方面，scrapy不修改原始页面html，因此如果使用 <code class="docutils literal notranslate"><span class="pre">&lt;tbody&gt;</span></code> 在xpath表达式中。</p>
<p>因此，您应该记住以下几点：</p>
<ul class="simple">
<li><p>检查DOM以查找要在Scrapy中使用的xpaths时禁用javascript（在“开发人员工具”设置中，单击 <cite>Disable JavaScript</cite> ）</p></li>
<li><p>不要使用完整的xpath路径，使用基于属性的相对路径和智能路径（例如 <code class="docutils literal notranslate"><span class="pre">id</span></code> ， <code class="docutils literal notranslate"><span class="pre">class</span></code> ， <code class="docutils literal notranslate"><span class="pre">width</span></code> 或任何识别特征，如 <code class="docutils literal notranslate"><span class="pre">contains(&#64;href,</span> <span class="pre">'image')</span></code> .</p></li>
<li><p>从不包括 <code class="docutils literal notranslate"><span class="pre">&lt;tbody&gt;</span></code> xpath表达式中的元素，除非您真正知道自己在做什么</p></li>
</ul>
</div>
<div class="section" id="inspecting-a-website">
<span id="topics-inspector"></span><h2>查看网站<a class="headerlink" href="#inspecting-a-website" title="永久链接至标题">¶</a></h2>
<p>到目前为止，开发人员工具最方便的特性是 <cite>Inspector</cite> feature, which allows you to inspect the underlying HTML code of any webpage. To demonstrate the Inspector, let's look at the <a class="reference external" href="http://quotes.toscrape.com">quotes.toscrape.com</a> 现场。</p>
<p>在这个网站上，我们总共有来自不同作者的十个引用，其中有特定的标签，还有前十个标签。假设我们想要提取这个页面上的所有引用，而不需要任何关于作者、标签等的元信息。</p>
<p>我们不必查看页面的整个源代码，只需右键单击一个报价并选择 <code class="docutils literal notranslate"><span class="pre">Inspect</span> <span class="pre">Element</span> <span class="pre">(Q)</span></code> 打开了 <cite>Inspector</cite> . 在里面你应该看到这样的东西：</p>
<a class="reference internal image-reference" href="../_images/inspector_01.png"><img alt="Firefox's Inspector-tool" src="../_images/inspector_01.png" style="width: 777px; height: 469px;" /></a>
<p>我们感兴趣的是：</p>
<div class="highlight-html notranslate"><div class="highlight"><pre><span></span><span class="p">&lt;</span><span class="nt">div</span> <span class="na">class</span><span class="o">=</span><span class="s">&quot;quote&quot;</span> <span class="na">itemscope</span><span class="o">=</span><span class="s">&quot;&quot;</span> <span class="na">itemtype</span><span class="o">=</span><span class="s">&quot;http://schema.org/CreativeWork&quot;</span><span class="p">&gt;</span>
  <span class="p">&lt;</span><span class="nt">span</span> <span class="na">class</span><span class="o">=</span><span class="s">&quot;text&quot;</span> <span class="na">itemprop</span><span class="o">=</span><span class="s">&quot;text&quot;</span><span class="p">&gt;</span>(...)<span class="p">&lt;/</span><span class="nt">span</span><span class="p">&gt;</span>
  <span class="p">&lt;</span><span class="nt">span</span><span class="p">&gt;</span>(...)<span class="p">&lt;/</span><span class="nt">span</span><span class="p">&gt;</span>
  <span class="p">&lt;</span><span class="nt">div</span> <span class="na">class</span><span class="o">=</span><span class="s">&quot;tags&quot;</span><span class="p">&gt;</span>(...)<span class="p">&lt;/</span><span class="nt">div</span><span class="p">&gt;</span>
<span class="p">&lt;/</span><span class="nt">div</span><span class="p">&gt;</span>
</pre></div>
</div>
<p>如果你在第一个上面徘徊 <code class="docutils literal notranslate"><span class="pre">div</span></code> 正上方 <code class="docutils literal notranslate"><span class="pre">span</span></code> 在屏幕截图中突出显示的标签，您将看到网页的相应部分也会突出显示。现在我们有了一个部分，但是我们在任何地方都找不到报价文本。</p>
<p>的优势 <cite>Inspector</cite> 它自动展开和折叠网页的部分和标签，大大提高了可读性。您可以通过单击标签前面的箭头或直接双击标签来展开和折叠标签。如果我们扩大 <code class="docutils literal notranslate"><span class="pre">span</span></code> 带标签 <code class="docutils literal notranslate"><span class="pre">class=</span> <span class="pre">&quot;text&quot;</span></code> 我们将看到我们单击的报价文本。这个 <cite>Inspector</cite> 允许您将xpath复制到选定的元素。让我们试试看。</p>
<p>首先在http://quotes.toscrape.com/在终端中：</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>$ scrapy shell &quot;http://quotes.toscrape.com/&quot;
</pre></div>
</div>
<p>然后，返回web浏览器，右键单击 <code class="docutils literal notranslate"><span class="pre">span</span></code> 选择标记 <code class="docutils literal notranslate"><span class="pre">Copy</span> <span class="pre">&gt;</span> <span class="pre">XPath</span></code> 然后把它贴在这个破壳里：</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">response</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;/html/body/div/div[2]/div[1]/div[1]/span[1]/text()&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">getall</span><span class="p">()</span>
<span class="go">[&#39;“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”&#39;]</span>
</pre></div>
</div>
<p>添加 <code class="docutils literal notranslate"><span class="pre">text()</span></code> 最后，我们可以用这个基本选择器提取第一个报价。但这个xpath并没有那么聪明。它所做的就是在源代码中沿着所需的路径从 <code class="docutils literal notranslate"><span class="pre">html</span></code> . 那么让我们看看我们是否可以改进一下xpath：</p>
<p>如果我们检查 <cite>Inspector</cite> 我们将再次看到，在我们的 <code class="docutils literal notranslate"><span class="pre">div</span></code> 标签我们有九个相同的 <code class="docutils literal notranslate"><span class="pre">div</span></code> 标签，每个标签都具有与第一个相同的属性。如果我们扩展其中任何一个，我们将看到与第一个报价相同的结构：两个 <code class="docutils literal notranslate"><span class="pre">span</span></code> 标签和一个 <code class="docutils literal notranslate"><span class="pre">div</span></code> 标签。我们可以扩大每个 <code class="docutils literal notranslate"><span class="pre">span</span></code> 带标签 <code class="docutils literal notranslate"><span class="pre">class=&quot;text&quot;</span></code> 在我们内部 <code class="docutils literal notranslate"><span class="pre">div</span></code> 标记并查看每个引用：</p>
<div class="highlight-html notranslate"><div class="highlight"><pre><span></span><span class="p">&lt;</span><span class="nt">div</span> <span class="na">class</span><span class="o">=</span><span class="s">&quot;quote&quot;</span> <span class="na">itemscope</span><span class="o">=</span><span class="s">&quot;&quot;</span> <span class="na">itemtype</span><span class="o">=</span><span class="s">&quot;http://schema.org/CreativeWork&quot;</span><span class="p">&gt;</span>
  <span class="p">&lt;</span><span class="nt">span</span> <span class="na">class</span><span class="o">=</span><span class="s">&quot;text&quot;</span> <span class="na">itemprop</span><span class="o">=</span><span class="s">&quot;text&quot;</span><span class="p">&gt;</span>
    “The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”
  <span class="p">&lt;/</span><span class="nt">span</span><span class="p">&gt;</span>
  <span class="p">&lt;</span><span class="nt">span</span><span class="p">&gt;</span>(...)<span class="p">&lt;/</span><span class="nt">span</span><span class="p">&gt;</span>
  <span class="p">&lt;</span><span class="nt">div</span> <span class="na">class</span><span class="o">=</span><span class="s">&quot;tags&quot;</span><span class="p">&gt;</span>(...)<span class="p">&lt;/</span><span class="nt">div</span><span class="p">&gt;</span>
<span class="p">&lt;/</span><span class="nt">div</span><span class="p">&gt;</span>
</pre></div>
</div>
<p>有了这些知识，我们可以改进我们的xpath：我们只需选择 <code class="docutils literal notranslate"><span class="pre">span</span></code> 标签与 <code class="docutils literal notranslate"><span class="pre">class=&quot;text&quot;</span></code> 通过使用 <a class="reference external" href="https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions">has-class-extension</a> ：</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">response</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;//span[has-class(&quot;text&quot;)]/text()&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">getall</span><span class="p">()</span>
<span class="go">[&#39;“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”&#39;,</span>
<span class="go">&#39;“It is our choices, Harry, that show what we truly are, far more than our abilities.”&#39;,</span>
<span class="go">&#39;“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”&#39;,</span>
<span class="go">...]</span>
</pre></div>
</div>
<p>通过一个简单、更聪明的xpath，我们能够从页面中提取所有的引号。我们可以在第一个xpath上构建一个循环，以增加最后一个xpath的数量。 <code class="docutils literal notranslate"><span class="pre">div</span></code> ，但这将不必要地复杂，只需使用 <code class="docutils literal notranslate"><span class="pre">has-class(&quot;text&quot;)</span></code> 我们能够在一行中提取所有报价。</p>
<p>这个 <cite>Inspector</cite> 还有很多其他有用的功能，比如在源代码中搜索或者直接滚动到您选择的元素。让我们演示一个用例：</p>
<p>说你想找到 <code class="docutils literal notranslate"><span class="pre">Next</span></code> 页面上的按钮。类型 <code class="docutils literal notranslate"><span class="pre">Next</span></code> 在搜索栏的右上角 <cite>Inspector</cite> . 你应该得到两个结果。第一个是 <code class="docutils literal notranslate"><span class="pre">li</span></code> 带标签 <code class="docutils literal notranslate"><span class="pre">class=&quot;next&quot;</span></code> ，第二个是 <code class="docutils literal notranslate"><span class="pre">a</span></code> 标签。右键单击 <code class="docutils literal notranslate"><span class="pre">a</span></code> 标记与选择 <code class="docutils literal notranslate"><span class="pre">Scroll</span> <span class="pre">into</span> <span class="pre">View</span></code> . 如果您将鼠标悬停在标签上，您将看到突出显示的按钮。从这里我们可以很容易地创建一个 <a class="reference internal" href="link-extractors.html#topics-link-extractors"><span class="std std-ref">Link Extractor</span></a> 跟随分页。在这样一个简单的站点上，可能不需要从视觉上查找元素，而是 <code class="docutils literal notranslate"><span class="pre">Scroll</span> <span class="pre">into</span> <span class="pre">View</span></code> 函数在复杂的站点上非常有用。</p>
<p>请注意，搜索栏也可用于搜索和测试CSS选择器。例如，您可以搜索 <code class="docutils literal notranslate"><span class="pre">span.text</span></code> 查找所有报价文本。而不是全文搜索，这将搜索 <code class="docutils literal notranslate"><span class="pre">span</span></code> 带标签 <code class="docutils literal notranslate"><span class="pre">class=&quot;text&quot;</span></code> 在页面中。</p>
</div>
<div class="section" id="the-network-tool">
<span id="topics-network-tool"></span><h2>网络工具<a class="headerlink" href="#the-network-tool" title="永久链接至标题">¶</a></h2>
<p>在抓取过程中，您可能会遇到动态网页，其中页面的某些部分是通过多个请求动态加载的。虽然这很棘手，但是 <cite>Network</cite>-tool in the Developer Tools greatly facilitates this task. To demonstrate the Network-tool, let's take a look at the page <a class="reference external" href="http://quotes.toscrape.com/scroll">quotes.toscrape.com/scroll</a> .</p>
<p>页面与基本页面非常相似 <a class="reference external" href="http://quotes.toscrape.com">quotes.toscrape.com</a> -第页，但不是上面提到的 <code class="docutils literal notranslate"><span class="pre">Next</span></code> 按钮，则当您滚动到底部时，页面会自动加载新的引号。我们可以直接尝试不同的xpath，但是我们将从Scrapy shell中检查另一个非常有用的命令：</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>$ scrapy shell &quot;quotes.toscrape.com/scroll&quot;
(...)
&gt;&gt;&gt; view(response)
</pre></div>
</div>
<p>浏览器窗口应该和网页一起打开，但有一个关键的区别：我们看到的不是引用，而是一个带单词的绿色条。 <code class="docutils literal notranslate"><span class="pre">Loading...</span></code> .</p>
<a class="reference internal image-reference" href="../_images/network_01.png"><img alt="Response from quotes.toscrape.com/scroll" src="../_images/network_01.png" style="width: 777px; height: 296px;" /></a>
<p>这个 <code class="docutils literal notranslate"><span class="pre">view(response)</span></code> 命令让我们查看shell或稍后蜘蛛从服务器接收到的响应。这里我们看到加载了一些基本模板，其中包括标题、登录按钮和页脚，但是缺少引号。这告诉我们报价是从不同的请求加载的，而不是 <code class="docutils literal notranslate"><span class="pre">quotes.toscrape/scroll</span></code> .</p>
<p>如果你点击 <code class="docutils literal notranslate"><span class="pre">Network</span></code> 选项卡，您可能只能看到两个条目。我们要做的第一件事是通过单击 <code class="docutils literal notranslate"><span class="pre">Persist</span> <span class="pre">Logs</span></code> . 如果禁用此选项，则每次导航到不同的页面时，日志都会自动清除。启用这个选项是一个很好的默认设置，因为它可以让我们控制何时清除日志。</p>
<p>如果我们现在重新加载页面，您将看到日志中填充了六个新的请求。</p>
<a class="reference internal image-reference" href="../_images/network_02.png"><img alt="Network tab with persistent logs and requests" src="../_images/network_02.png" style="width: 777px; height: 241px;" /></a>
<p>在这里，我们可以看到在重新加载页面时发出的每个请求，并且可以检查每个请求及其响应。因此，让我们找出我们的报价来自哪里：</p>
<p>首先单击带有名称的请求 <code class="docutils literal notranslate"><span class="pre">scroll</span></code> . 在右边，您现在可以检查请求。在 <code class="docutils literal notranslate"><span class="pre">Headers</span></code> 您将找到有关请求头的详细信息，例如URL、方法、IP地址等。我们将忽略其他选项卡并直接单击 <code class="docutils literal notranslate"><span class="pre">Response</span></code> .</p>
<p>你应该在里面看到什么 <code class="docutils literal notranslate"><span class="pre">Preview</span></code> 窗格是呈现的HTML代码，这正是我们调用 <code class="docutils literal notranslate"><span class="pre">view(response)</span></code> 在贝壳里。相应地 <code class="docutils literal notranslate"><span class="pre">type</span></code> 日志中的请求为 <code class="docutils literal notranslate"><span class="pre">html</span></code> . 其他请求的类型如下 <code class="docutils literal notranslate"><span class="pre">css</span></code> 或 <code class="docutils literal notranslate"><span class="pre">js</span></code> 但是我们感兴趣的是一个要求 <code class="docutils literal notranslate"><span class="pre">quotes?page=1</span></code> 与类型 <code class="docutils literal notranslate"><span class="pre">json</span></code> .</p>
<p>如果我们点击这个请求，我们会看到请求的URL是 <code class="docutils literal notranslate"><span class="pre">http://quotes.toscrape.com/api/quotes?page=1</span></code> 响应是一个包含我们的引号的JSON对象。我们也可以右键单击请求并打开 <code class="docutils literal notranslate"><span class="pre">Open</span> <span class="pre">in</span> <span class="pre">new</span> <span class="pre">tab</span></code> 以获得更好的概述。</p>
<a class="reference internal image-reference" href="../_images/network_03.png"><img alt="JSON-object returned from the quotes.toscrape API" src="../_images/network_03.png" style="width: 777px; height: 375px;" /></a>
<p>有了这个响应，我们现在可以轻松地解析JSON对象，并请求每个页面获取站点上的每个引用：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">scrapy</span>
<span class="kn">import</span> <span class="nn">json</span>


<span class="k">class</span> <span class="nc">QuoteSpider</span><span class="p">(</span><span class="n">scrapy</span><span class="o">.</span><span class="n">Spider</span><span class="p">):</span>
    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;quote&#39;</span>
    <span class="n">allowed_domains</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;quotes.toscrape.com&#39;</span><span class="p">]</span>
    <span class="n">page</span> <span class="o">=</span> <span class="mi">1</span>
    <span class="n">start_urls</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;http://quotes.toscrape.com/api/quotes?page=1&#39;</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">response</span><span class="p">):</span>
        <span class="n">data</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
        <span class="k">for</span> <span class="n">quote</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="s2">&quot;quotes&quot;</span><span class="p">]:</span>
            <span class="k">yield</span> <span class="p">{</span><span class="s2">&quot;quote&quot;</span><span class="p">:</span> <span class="n">quote</span><span class="p">[</span><span class="s2">&quot;text&quot;</span><span class="p">]}</span>
        <span class="k">if</span> <span class="n">data</span><span class="p">[</span><span class="s2">&quot;has_next&quot;</span><span class="p">]:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">page</span> <span class="o">+=</span> <span class="mi">1</span>
            <span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;http://quotes.toscrape.com/api/quotes?page=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">page</span><span class="si">}</span><span class="s2">&quot;</span>
            <span class="k">yield</span> <span class="n">scrapy</span><span class="o">.</span><span class="n">Request</span><span class="p">(</span><span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">,</span> <span class="n">callback</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">parse</span><span class="p">)</span>
</pre></div>
</div>
<p>这个蜘蛛程序从QuotesAPI的第一页开始。对于每个响应，我们分析 <code class="docutils literal notranslate"><span class="pre">response.text</span></code> 并分配给 <code class="docutils literal notranslate"><span class="pre">data</span></code> . 这让我们可以像在Python字典上一样对JSON对象进行操作。我们迭代 <code class="docutils literal notranslate"><span class="pre">quotes</span></code> 打印出 <code class="docutils literal notranslate"><span class="pre">quote[&quot;text&quot;]</span></code> . 如果方便的话 <code class="docutils literal notranslate"><span class="pre">has_next</span></code> 元素是 <code class="docutils literal notranslate"><span class="pre">true</span></code> （尝试加载 <a class="reference external" href="http://quotes.toscrape.com/api/quotes?page=10">quotes.toscrape.com/api/quotes?page=10</a> 在您的浏览器或大于10的页码中，我们增加 <code class="docutils literal notranslate"><span class="pre">page</span></code> 属性与 <code class="docutils literal notranslate"><span class="pre">yield</span></code> 一个新的请求，将递增的页码插入到 <code class="docutils literal notranslate"><span class="pre">url</span></code> .</p>
<p id="requests-from-curl">在更复杂的网站中，很难轻松地复制请求，因为我们需要添加 <code class="docutils literal notranslate"><span class="pre">headers</span></code> 或 <code class="docutils literal notranslate"><span class="pre">cookies</span></code> 让它发挥作用。在这些情况下，您可以在中导出请求 <a class="reference external" href="https://curl.haxx.se/">cURL</a> 格式化，在网络工具中右键单击它们并使用 <a class="reference internal" href="request-response.html#scrapy.http.Request.from_curl" title="scrapy.http.Request.from_curl"><code class="xref py py-meth docutils literal notranslate"><span class="pre">from_curl()</span></code></a> 方法生成等效请求：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">scrapy</span> <span class="kn">import</span> <span class="n">Request</span>

<span class="n">request</span> <span class="o">=</span> <span class="n">Request</span><span class="o">.</span><span class="n">from_curl</span><span class="p">(</span>
    <span class="s2">&quot;curl &#39;http://quotes.toscrape.com/api/quotes?page=1&#39; -H &#39;User-Agent: Mozil&quot;</span>
    <span class="s2">&quot;la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0&#39; -H &#39;Acce&quot;</span>
    <span class="s2">&quot;pt: */*&#39; -H &#39;Accept-Language: ca,en-US;q=0.7,en;q=0.3&#39; --compressed -H &#39;X&quot;</span>
    <span class="s2">&quot;-Requested-With: XMLHttpRequest&#39; -H &#39;Proxy-Authorization: Basic QFRLLTAzM&quot;</span>
    <span class="s2">&quot;zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW&quot;</span>
    <span class="s2">&quot;I0LWJlZGYtMmM0YjgzZmI0MGY0&#39; -H &#39;Connection: keep-alive&#39; -H &#39;Referer: http&quot;</span>
    <span class="s2">&quot;://quotes.toscrape.com/scroll&#39; -H &#39;Cache-Control: max-age=0&#39;&quot;</span><span class="p">)</span>
</pre></div>
</div>
<p>或者，如果您想知道重新创建该请求所需的参数，可以使用 <a class="reference internal" href="#scrapy.utils.curl.curl_to_request_kwargs" title="scrapy.utils.curl.curl_to_request_kwargs"><code class="xref py py-func docutils literal notranslate"><span class="pre">curl_to_request_kwargs()</span></code></a> 函数获取具有等效参数的字典：</p>
<dl class="py function">
<dt id="scrapy.utils.curl.curl_to_request_kwargs">
<code class="sig-prename descclassname">scrapy.utils.curl.</code><code class="sig-name descname">curl_to_request_kwargs</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">curl_command</span></em>, <em class="sig-param"><span class="n">ignore_unknown_options</span><span class="o">=</span><span class="default_value">True</span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/scrapy/utils/curl.html#curl_to_request_kwargs"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.utils.curl.curl_to_request_kwargs" title="永久链接至目标">¶</a></dt>
<dd><p>将cURL命令语法转换为请求kwargs。</p>
<dl class="field-list simple">
<dt class="field-odd">参数</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>curl_command</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(在 Python v3.9)"><em>str</em></a>) -- 包含curl命令的字符串</p></li>
<li><p><strong>ignore_unknown_options</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(在 Python v3.9)"><em>bool</em></a>) -- 如果为true，则当cURL选项未知时仅发出警告。否则将引发错误。（默认值：True）</p></li>
</ul>
</dd>
<dt class="field-even">返回</dt>
<dd class="field-even"><p>请求字典</p>
</dd>
</dl>
</dd></dl>

<p>注意，要将cURL命令转换为Scrapy请求，可以使用 <a class="reference external" href="https://michael-shub.github.io/curl2scrapy/">curl2scrapy</a> .</p>
<p>如你所见，在 <cite>Network</cite> -工具我们能够轻松地复制页面滚动功能的动态请求。对动态页面进行爬行可能非常困难，页面也可能非常复杂，但是（主要）归根结底就是识别正确的请求并在蜘蛛中复制它。</p>
</div>
</div>


           </div>
           
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="dynamic-content.html" class="btn btn-neutral float-right" title="选择动态加载的内容" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="broad-crawls.html" class="btn btn-neutral float-left" title="宽爬行" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        
        &copy; 版权所有 2008–2020, Scrapy developers
      <span class="lastupdated">
        最后更新于 10月 18, 2020.
      </span>

    </p>
  </div>
    
    
    
    Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a
    
    <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a>
    
    provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
  
 
<script type="text/javascript">
!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t){var e=document.createElement("script");e.type="text/javascript";e.async=!0;e.src=("https:"===document.location.protocol?"https://":"http://")+"cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(e,n)};analytics.SNIPPET_VERSION="3.1.0";
analytics.load("8UDQfnf3cyFSTsM4YANnW5sXmgZVILbA");
analytics.page();
}}();

analytics.ready(function () {
    ga('require', 'linker');
    ga('linker:autoLink', ['scrapinghub.com', 'crawlera.com']);
});
</script>


</body>
</html>